{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import altair as alt\n",
    "import shap\n",
    "\n",
    "from marginal import MarginalExplainer\n",
    "import plot\n",
    "import matplotlib.pyplot as plt\n",
    "import utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = 50000\n",
    "lamb = 0.5\n",
    "d = (8, 8, 3)\n",
    "epochs = 50\n",
    "hidden_layers = 64\n",
    "batch_size = 50\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.random.randn(n, *d).astype(np.float32)\n",
    "y = (np.sum(X[:, 1, :, 1], axis=-1) > 0).astype(np.float32) + \\\n",
    "    (np.sum(X[:, 1, :, 1], axis=-1) > 1).astype(np.float32) + \\\n",
    "    np.sum(X[:, 0, :, 0], axis=-1) * lamb\n",
    "\n",
    "y = y.astype(np.float32)\n",
    "\n",
    "X_train = X[:int(n * 0.8)]\n",
    "y_train = y[:int(n * 0.8)]\n",
    "X_val   = X[int(n * 0.8):]\n",
    "y_val   = y[int(n * 0.8):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.Sequential()\n",
    "model.add(tf.keras.Input(shape=d, batch_size=batch_size))\n",
    "model.add(tf.keras.layers.Flatten())\n",
    "model.add(tf.keras.layers.Dense(hidden_layers, activation=tf.keras.activations.relu, use_bias=True))\n",
    "model.add(tf.keras.layers.Dense(hidden_layers, activation=tf.keras.activations.relu, use_bias=True))\n",
    "model.add(tf.keras.layers.Dense(hidden_layers, activation=tf.keras.activations.relu, use_bias=True))\n",
    "model.add(tf.keras.layers.Dense(1, activation=None, use_bias=False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate)\n",
    "model.compile(optimizer=optimizer,\n",
    "              loss=tf.keras.losses.MSE,\n",
    "              metrics=[tf.keras.metrics.MeanAbsoluteError()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/50\n",
      "40000/40000 - 2s - loss: 2.5415 - mean_absolute_error: 1.2839 - val_loss: 2.0170 - val_mean_absolute_error: 1.1424\n",
      "Epoch 2/50\n",
      "40000/40000 - 2s - loss: 1.3474 - mean_absolute_error: 0.9260 - val_loss: 0.8835 - val_mean_absolute_error: 0.7513\n",
      "Epoch 3/50\n",
      "40000/40000 - 2s - loss: 0.7237 - mean_absolute_error: 0.6825 - val_loss: 0.6357 - val_mean_absolute_error: 0.6395\n",
      "Epoch 4/50\n",
      "40000/40000 - 2s - loss: 0.5790 - mean_absolute_error: 0.6117 - val_loss: 0.5513 - val_mean_absolute_error: 0.5967\n",
      "Epoch 5/50\n",
      "40000/40000 - 2s - loss: 0.5146 - mean_absolute_error: 0.5774 - val_loss: 0.5050 - val_mean_absolute_error: 0.5719\n",
      "Epoch 6/50\n",
      "40000/40000 - 2s - loss: 0.4736 - mean_absolute_error: 0.5546 - val_loss: 0.4721 - val_mean_absolute_error: 0.5540\n",
      "Epoch 7/50\n",
      "40000/40000 - 2s - loss: 0.4437 - mean_absolute_error: 0.5378 - val_loss: 0.4482 - val_mean_absolute_error: 0.5401\n",
      "Epoch 8/50\n",
      "40000/40000 - 2s - loss: 0.4206 - mean_absolute_error: 0.5242 - val_loss: 0.4293 - val_mean_absolute_error: 0.5291\n",
      "Epoch 9/50\n",
      "40000/40000 - 2s - loss: 0.4022 - mean_absolute_error: 0.5132 - val_loss: 0.4145 - val_mean_absolute_error: 0.5204\n",
      "Epoch 10/50\n",
      "40000/40000 - 2s - loss: 0.3867 - mean_absolute_error: 0.5037 - val_loss: 0.4009 - val_mean_absolute_error: 0.5123\n",
      "Epoch 11/50\n",
      "40000/40000 - 2s - loss: 0.3737 - mean_absolute_error: 0.4957 - val_loss: 0.3898 - val_mean_absolute_error: 0.5051\n",
      "Epoch 12/50\n",
      "40000/40000 - 2s - loss: 0.3628 - mean_absolute_error: 0.4886 - val_loss: 0.3802 - val_mean_absolute_error: 0.4992\n",
      "Epoch 13/50\n",
      "40000/40000 - 2s - loss: 0.3529 - mean_absolute_error: 0.4821 - val_loss: 0.3744 - val_mean_absolute_error: 0.4951\n",
      "Epoch 14/50\n",
      "40000/40000 - 2s - loss: 0.3444 - mean_absolute_error: 0.4767 - val_loss: 0.3650 - val_mean_absolute_error: 0.4893\n",
      "Epoch 15/50\n",
      "40000/40000 - 2s - loss: 0.3370 - mean_absolute_error: 0.4720 - val_loss: 0.3577 - val_mean_absolute_error: 0.4853\n",
      "Epoch 16/50\n",
      "40000/40000 - 2s - loss: 0.3302 - mean_absolute_error: 0.4673 - val_loss: 0.3523 - val_mean_absolute_error: 0.4813\n",
      "Epoch 17/50\n",
      "40000/40000 - 2s - loss: 0.3241 - mean_absolute_error: 0.4633 - val_loss: 0.3464 - val_mean_absolute_error: 0.4775\n",
      "Epoch 18/50\n",
      "40000/40000 - 2s - loss: 0.3185 - mean_absolute_error: 0.4595 - val_loss: 0.3432 - val_mean_absolute_error: 0.4751\n",
      "Epoch 19/50\n",
      "40000/40000 - 2s - loss: 0.3135 - mean_absolute_error: 0.4559 - val_loss: 0.3376 - val_mean_absolute_error: 0.4716\n",
      "Epoch 20/50\n",
      "40000/40000 - 2s - loss: 0.3086 - mean_absolute_error: 0.4527 - val_loss: 0.3336 - val_mean_absolute_error: 0.4692\n",
      "Epoch 21/50\n",
      "40000/40000 - 2s - loss: 0.3042 - mean_absolute_error: 0.4496 - val_loss: 0.3298 - val_mean_absolute_error: 0.4662\n",
      "Epoch 22/50\n",
      "40000/40000 - 2s - loss: 0.3004 - mean_absolute_error: 0.4469 - val_loss: 0.3272 - val_mean_absolute_error: 0.4645\n",
      "Epoch 23/50\n",
      "40000/40000 - 2s - loss: 0.2966 - mean_absolute_error: 0.4441 - val_loss: 0.3231 - val_mean_absolute_error: 0.4619\n",
      "Epoch 24/50\n",
      "40000/40000 - 2s - loss: 0.2931 - mean_absolute_error: 0.4416 - val_loss: 0.3200 - val_mean_absolute_error: 0.4598\n",
      "Epoch 25/50\n",
      "40000/40000 - 2s - loss: 0.2899 - mean_absolute_error: 0.4394 - val_loss: 0.3177 - val_mean_absolute_error: 0.4578\n",
      "Epoch 26/50\n",
      "40000/40000 - 2s - loss: 0.2869 - mean_absolute_error: 0.4371 - val_loss: 0.3163 - val_mean_absolute_error: 0.4569\n",
      "Epoch 27/50\n",
      "40000/40000 - 2s - loss: 0.2841 - mean_absolute_error: 0.4351 - val_loss: 0.3123 - val_mean_absolute_error: 0.4544\n",
      "Epoch 28/50\n",
      "40000/40000 - 2s - loss: 0.2814 - mean_absolute_error: 0.4330 - val_loss: 0.3111 - val_mean_absolute_error: 0.4534\n",
      "Epoch 29/50\n",
      "40000/40000 - 2s - loss: 0.2788 - mean_absolute_error: 0.4312 - val_loss: 0.3080 - val_mean_absolute_error: 0.4513\n",
      "Epoch 30/50\n",
      "40000/40000 - 2s - loss: 0.2766 - mean_absolute_error: 0.4295 - val_loss: 0.3063 - val_mean_absolute_error: 0.4499\n",
      "Epoch 31/50\n",
      "40000/40000 - 2s - loss: 0.2742 - mean_absolute_error: 0.4276 - val_loss: 0.3044 - val_mean_absolute_error: 0.4490\n",
      "Epoch 32/50\n",
      "40000/40000 - 2s - loss: 0.2720 - mean_absolute_error: 0.4260 - val_loss: 0.3029 - val_mean_absolute_error: 0.4478\n",
      "Epoch 33/50\n",
      "40000/40000 - 2s - loss: 0.2700 - mean_absolute_error: 0.4245 - val_loss: 0.3008 - val_mean_absolute_error: 0.4466\n",
      "Epoch 34/50\n",
      "40000/40000 - 2s - loss: 0.2680 - mean_absolute_error: 0.4229 - val_loss: 0.2990 - val_mean_absolute_error: 0.4451\n",
      "Epoch 35/50\n",
      "40000/40000 - 2s - loss: 0.2663 - mean_absolute_error: 0.4216 - val_loss: 0.2982 - val_mean_absolute_error: 0.4444\n",
      "Epoch 36/50\n",
      "40000/40000 - 2s - loss: 0.2645 - mean_absolute_error: 0.4202 - val_loss: 0.2961 - val_mean_absolute_error: 0.4431\n",
      "Epoch 37/50\n",
      "40000/40000 - 2s - loss: 0.2627 - mean_absolute_error: 0.4188 - val_loss: 0.2953 - val_mean_absolute_error: 0.4422\n",
      "Epoch 38/50\n",
      "40000/40000 - 2s - loss: 0.2611 - mean_absolute_error: 0.4176 - val_loss: 0.2953 - val_mean_absolute_error: 0.4424\n",
      "Epoch 39/50\n",
      "40000/40000 - 2s - loss: 0.2594 - mean_absolute_error: 0.4164 - val_loss: 0.2929 - val_mean_absolute_error: 0.4401\n",
      "Epoch 40/50\n",
      "40000/40000 - 2s - loss: 0.2581 - mean_absolute_error: 0.4152 - val_loss: 0.2910 - val_mean_absolute_error: 0.4394\n",
      "Epoch 41/50\n",
      "40000/40000 - 2s - loss: 0.2566 - mean_absolute_error: 0.4140 - val_loss: 0.2900 - val_mean_absolute_error: 0.4387\n",
      "Epoch 42/50\n",
      "40000/40000 - 2s - loss: 0.2552 - mean_absolute_error: 0.4129 - val_loss: 0.2887 - val_mean_absolute_error: 0.4377\n",
      "Epoch 43/50\n",
      "40000/40000 - 2s - loss: 0.2538 - mean_absolute_error: 0.4119 - val_loss: 0.2880 - val_mean_absolute_error: 0.4370\n",
      "Epoch 44/50\n",
      "40000/40000 - 2s - loss: 0.2525 - mean_absolute_error: 0.4107 - val_loss: 0.2866 - val_mean_absolute_error: 0.4365\n",
      "Epoch 45/50\n",
      "40000/40000 - 2s - loss: 0.2513 - mean_absolute_error: 0.4098 - val_loss: 0.2856 - val_mean_absolute_error: 0.4355\n",
      "Epoch 46/50\n",
      "40000/40000 - 2s - loss: 0.2500 - mean_absolute_error: 0.4087 - val_loss: 0.2848 - val_mean_absolute_error: 0.4348\n",
      "Epoch 47/50\n",
      "40000/40000 - 2s - loss: 0.2488 - mean_absolute_error: 0.4077 - val_loss: 0.2847 - val_mean_absolute_error: 0.4345\n",
      "Epoch 48/50\n",
      "40000/40000 - 2s - loss: 0.2475 - mean_absolute_error: 0.4067 - val_loss: 0.2828 - val_mean_absolute_error: 0.4336\n",
      "Epoch 49/50\n",
      "40000/40000 - 2s - loss: 0.2466 - mean_absolute_error: 0.4059 - val_loss: 0.2818 - val_mean_absolute_error: 0.4330\n",
      "Epoch 50/50\n",
      "40000/40000 - 2s - loss: 0.2453 - mean_absolute_error: 0.4049 - val_loss: 0.2812 - val_mean_absolute_error: 0.4322\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7fab20604828>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs=epochs, verbose=2, batch_size=batch_size, validation_data=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_shap_samples=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/10 [00:00<?, ?it/s]\u001b[A\n",
      " 10%|█         | 1/10 [00:00<00:05,  1.61it/s]\u001b[A\n",
      " 20%|██        | 2/10 [00:01<00:04,  1.62it/s]\u001b[A\n",
      " 30%|███       | 3/10 [00:01<00:04,  1.63it/s]\u001b[A\n",
      " 40%|████      | 4/10 [00:02<00:03,  1.65it/s]\u001b[A\n",
      " 50%|█████     | 5/10 [00:03<00:03,  1.66it/s]\u001b[A\n",
      " 60%|██████    | 6/10 [00:03<00:02,  1.66it/s]\u001b[A\n",
      " 70%|███████   | 7/10 [00:04<00:01,  1.67it/s]\u001b[A\n",
      " 80%|████████  | 8/10 [00:04<00:01,  1.67it/s]\u001b[A\n",
      " 90%|█████████ | 9/10 [00:05<00:00,  1.67it/s]\u001b[A\n",
      "100%|██████████| 10/10 [00:06<00:00,  1.67it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "primal_explainer = MarginalExplainer(model, np.zeros((1, 8, 8, 3)).astype(np.float32), nsamples=200, representation='mobius')\n",
    "primal_effects = primal_explainer.explain(X[:num_shap_samples], verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/10 [00:00<?, ?it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 10%|█         | 1/10 [00:00<00:02,  3.46it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 20%|██        | 2/10 [00:00<00:02,  3.46it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 30%|███       | 3/10 [00:00<00:02,  3.46it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 40%|████      | 4/10 [00:01<00:01,  3.44it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 50%|█████     | 5/10 [00:01<00:01,  3.42it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 60%|██████    | 6/10 [00:01<00:01,  3.41it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 70%|███████   | 7/10 [00:02<00:00,  3.41it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 80%|████████  | 8/10 [00:02<00:00,  3.43it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      " 90%|█████████ | 9/10 [00:02<00:00,  3.43it/s]\u001b[A/home/psturm/.local/lib/python3.6/site-packages/shap/explainers/kernel.py:535: UserWarning: l1_reg=\"auto\" is deprecated and in the next version (v0.29) the behavior will change from a conditional use of AIC to simply \"num_features(10)\"!\n",
      "  \"l1_reg=\\\"auto\\\" is deprecated and in the next version (v0.29) the behavior will change from a \" \\\n",
      "\n",
      "100%|██████████| 10/10 [00:02<00:00,  3.41it/s]\u001b[A"
     ]
    }
   ],
   "source": [
    "model_func = lambda x: model(x.reshape(x.shape[0], 8, 8, 3).astype(np.float32)).numpy()\n",
    "kernel_explainer = shap.KernelExplainer(model_func, np.zeros((1, 192)))\n",
    "shap_values = kernel_explainer.shap_values(X[:num_shap_samples].reshape(num_shap_samples, -1), num_samples=200)\n",
    "shap_values = np.reshape(shap_values, (10, 8, 8, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7faacc231400>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAC7VJREFUeJzt3V2IXOUdx/HfbzcGmxoV+2LTbNpEkIC9igSL2JZWscQaqoVeGNALKeRKUUqRtFftRelNEXtRBIlv0FRbfAERqxUqtUK1JjFS82JJg20StTFY68tF4+78e7ETTGOSObPnOc/M/PP9wMLO5Ow5/5Pd355nZv/neRwRApDT1KgLANAdAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwILFFXez0PDtmutjxCSyudBxJ+nfFY0l1f/vWPNZsxWPVtqTScQ5KejvCg7brJOAzkh7vZM8f94XZgedYzG9U71iStFS9asda4krfMEmHo17Eo95pSZLWVDq17zTcjiE6kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHEGgXc9jrbr9rea3tT10UBKGNgwG1PS/qlpKskXSRpg+2Lui4MQHtNruCXSNobEfsi4oikByVd021ZAEpoEvDlkvYf8/hA/zkAY67Ym2y2N9reanvr26V2CqCVJgE/KGnFMY9n+s/9n4i4KyLWRsTa80pVB6CVJgF/UdKFtlfZXizpOkmPdVsWgBIG3i0bEbO2b5L0lKRpSfdExM7OKwPQWqPb4SPiCUlPdFwLgMLoZAMSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kJgjovxOP++Y2lh8tyfU+7DiaiOH6h1KknR+xd+/a+aqHWrquXrn1Xu9/M/3KV1Q6Xj3SfHG4KWLuIIDiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJNVnZ5B7bh2y/UqMgAOU0uYLfJ2ldx3UA6MDAgEfEs5JYywCYQLwGBxJrNG1yE7Y3Spq/h+ycUnsF0EaxK/ixSxdpSam9AmiDITqQWJM/kz0g6c+SVts+YPt73ZcFoIQma5NtqFEIgPIYogOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSKybpYvsWFRpRaHZmivTVFwlSZLser9/o9erdqzMav2IhKQIli4CTmsEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSKzJpIsrbD9je5ftnbZvqVEYgPYG9qLbXiZpWURst71U0jZJ10bErlN8Db3oJQ5HL/rEmbhe9Ih4IyK29z9/T9JuSctbVwigc0MtXWR7paQ1kl44wb99tHQRgLHQ+HZR22dJ+qOkn0bEIwO2ZYhe4nAM0SfOxA3RJcn2GZIelrRlULgBjI8m76Jb0t2SdkfE7d2XBKCUJlfwyyTdIOly2zv6H9/quC4ABTBl0zB4DY4BJvI1OIDJRMCBxAg4kBgBBxIj4EBiBBxIjIADiRFwILGh7iZrzNLs4k72/HFHKh1Hmu8uqHm4qNh8Ml3xUHP1OoZ603W7k2JuvBqGuIIDiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJNZl08Uzbf7H9cn/pop/UKAxAe01aVf8r6fKIeL8/ffJztn8XEc93XBuAlgYGPOZnZXy///CM/kflrmwAC9F04YNp2zskHZL0dESccOki21ttbyX+wHgYatpk2+dKelTSzRHxykm3m3KIu8kmC3eTFVHzbrLi0yZHxDuSnpG0bqFFAainybvon+lfuWX7E5KulLSn68IAtNfkXfRlku63Pa35Xwi/jYjHuy0LQAndLF3Ea/DJw2vwIib6NTiAyULAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCCxbhpd7MF/gS9kaqpeI0ODvoKyx1PNpomqnS7VDjVV71CSpJ4rXTOjR6MLcLoj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQGAEHEmsc8P7c6C/ZZj42YEIMcwW/RdLurgoBUF7TlU1mJF0taXO35QAoqekV/A5Jt0kV734A0FqThQ/WSzoUEdsGbPfR2mQAxsLA20Vt/0zSDZJmJZ0p6WxJj0TE9af4Gm4XLXE8bhdt7XS/XXTYxQe/LukHEbF+wHYEvMTxCHhrp3vA+Ts4kBgzugyBK3ghXMHb4woOgIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGKLutmtFV3t+jhzvdkqx5Gkade9ma5Xvgfp5LI2n9TtTZKi1jWz2c8iV3AgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCCxRu1mtl+T9J6kOUmzEbG2y6IAlDFMP+k3IuJwZ5UAKI4hOpBY04CHpN/b3mZ7Y5cFASin6RD9KxFx0PZnJT1te09EPHvsBv3gE35gjAw9L7rtH0t6PyJ+fvJtpqKzO1E/pubtojXv35Tmqt4uWu9QmW8X7UW9n/si86Lb/qTtpUc/l/RNSa+0LxBA15r8ujlf0qO2j27/64h4stOqABTR0dJFDNFLYIheAEN0AFkRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIrKO/yoekD7vZ9fGmKnZo9Gp2aKhqk0bN5pOqC0DV7U2SKzVeNT0truBAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBijQJu+1zbD9neY3u37Uu7LgxAe01bVX8h6cmI+K7txZKWdFgTgEIGTrpo+xxJOyRdEA1naLQrzk5YsRd9unIv+lzNXvSKPdtVe9Erq/UtC6nYpIurJL0l6V7bL9ne3J8fHcCYaxLwRZIulnRnRKyR9IGkTcdvZHuj7a22txauEcACNRmif07S8xGxsv/4q5I2RcTVp/gahugFMESfPBM3RI+INyXtt726/9QVkna1Kw9ADU3fRb9Z0pb+O+j7JN3YXUkASulo6SKG6CUwRJ88EzdEBzC5CDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQ6WpvM3e36OFO9SmugSZpTxXXQpLoLhlVedq2WmkvXSVKtHs65hu2AXMGBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHEBgbc9mrbO475eNf2rTWKA9DOUJMu2p6WdFDSlyPiHyffbiqqtaqqXqtqr3ar6nS9/tGaXbE1J13M3KraxaSLV0j6+6nCDWB8DHuZvU7SAyf6B9sbJW1sXRGAYhoP0fuLHrwu6UsR8a9Tb8sQvQiG6K0xRG/uKknbB4UbwPgYJuAbdJLhOYDx1GiI3l8P/J+SLoiI/wzeniF6EQzRWzvdh+iNUhgRH0j6VOuqAFRFJxuQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQ6ajeLw9KHw95S+mlJh4c9Us2uqBbr+yzo3GouJ7TA/8eFnVdFvYX9H479eUn6YpONhprwoUu2t0bE2lHX0YWs58Z5jT+G6EBiBBxIbJwCfteoC+hQ1nPjvMbc2LwGB1DeOF3BARQ2FgG3vc72q7b32t406npKsL3C9jO2d9neafuWUddUku1p2y/ZfnzUtZRk+1zbD9neY3u37UtHXVMbIx+i9+da/5ukKyUdkPSipA0RsWukhbVke5mkZRGx3fZSSdskXTvp53WU7e9LWivp7IhYP+p6SrF9v6Q/RcTm/kSjSyLinVHXtVDjcAW/RNLeiNgXEUckPSjpmhHX1FpEvBER2/ufvydpt6Tlo62qDNszkq6WtHnUtZRk+xxJX5N0tyRFxJFJDrc0HgFfLmn/MY8PKEkQjrK9UtIaSS+MtpJi7pB0m2o3EnZvlaS3JN3bf/mxuT8f4cQah4CnZvssSQ9LujUi3h11PW3ZXi/pUERsG3UtHVgk6WJJd0bEGkkfSJro94TGIeAHJa045vFM/7mJZ/sMzYd7S0Q8Mup6CrlM0rdtv6b5l1OX2/7VaEsq5oCkAxFxdKT1kOYDP7HGIeAvSrrQ9qr+mxrXSXpsxDW1Ztuafy23OyJuH3U9pUTEDyNiJiJWav579YeIuH7EZRUREW9K2m97df+pKyRN9JuidSYvP4WImLV9k6SnJE1Luicido64rBIuk3SDpL/a3tF/7kcR8cQIa8JgN0va0r/Y7JN044jraWXkfyYD0J1xGKID6AgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHE/gedxkfQyJqxZAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "shap_values_mean_abs = np.mean(np.abs(shap_values), axis=0)\n",
    "plt.imshow(utils.normalize(shap_values_mean_abs, _range=[0.0, 1.0], _domain=[0.0, shap_values_mean_abs.max()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W1101 14:52:43.805624 140375857694528 image.py:648] Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7faa9459dba8>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADDZJREFUeJzt3VusXGUZxvHnmdm7VI5NLBjSNrYa0ogXpqTBEIxRCKZII154QRO4ICa9gkDUEPRKL4wxUYIXhoQUkMQKMQUSQhAkkYhERdpSlR4wtUG6OVga5Rjs7ux5vdhTLdB21nSt9c3st/9fspM909X53snMM9+atb+DI0IAcuqMuwAA7SHgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSm2rjQZfasbKNBx6znly0vVmXG2V4esHP+rnEoye7hV6zF0M6GDH0DdlKwFdK2jpVKAz9Ms1I0gFNl2tM0kz3cLG2LtLiYm292S/3vEp/lixZNFeknbWHqr3xOUUHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJBYpYDbXmf7Bdt7bd/adlEAmjE04La7kn4q6UpJF0raYPvCtgsDUF+VHvxiSXsjYl9EzEq6X9LV7ZYFoAlVAr5M0v6jbs8M7gMw4Rq7yGZ7o+2ttre+3tSDAqilSsBflrTiqNvLB/e9T0TcGRFrI2LtuU1VB6CWKgF/VtIFtlfZXiTpGkkPt1sWgCYMnQ8eET3bN0h6XFJX0t0RsbP1ygDUVmnBh4h4VNKjLdcCoGGMZAMSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kJijha0f/MludH9YZqeM/iuHirQjSYveK7iNiqS5peXaik8V/Kz/fbmm9O+yfZjPa2WzoA/p/fg/ipf6Q7cPogcHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSq7Kzyd22D9h+vkRBAJpTpQf/maR1LdcBoAVDAx4RT0n6V4FaADSM7+BAYo1NfbG9UdJGSdLSoZNcABTQWA9+9NZFOpuAA5OAU3QgsSp/JrtP0h8krbY9Y/vr7ZcFoAlV9ibbUKIQAM3jFB1IjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGKtbF3UseO0qTKfHb2CH1Hd4TvFNKpTsLlDBT/rY65YU1qsgo1J6qv5PB3LbD/Uj2DrIuBURsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHEqiy6uML2k7Z32d5p+6YShQGor8rGBz1J34yI7bbPkrTN9hMRsavl2gDUVGVvslcjYvvg97cl7Za0rO3CANQ30tZFtldKWiPpmWP82/+3LgIwESpPF7V9pqTfSvp+RDx4omOZLtoMpovWx3TRCmxPS3pA0uZh4QYwOapcRbekuyTtjojb2i8JQFOq9OCXSrpO0mW2dwx+vtxyXQAaUGVvsqclsR8wsAAxkg1IjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGIjzSarKizNFhoa03G3TEOSeuoXa0uS+tNlJi5IUudQued2ZsF+5b0o24d5UaH2Zg9XOoweHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSKzKoouLbf/J9p8HWxd9r0RhAOqrMlT1kKTLIuKdwfLJT9v+VUT8seXaANRUZdHFkPTO4Ob04KfcIGkAJ63qxgdd2zskHZD0REQcc+si21ttbyX+wGSovHWRJNleIukhSTdGxPPHPa7j6BTauqjTKTebLOYKzyZbVHI2WbmVsc8oOMPrvRHe300oNZvs8Oxh9fvD99IaqZqIeEPSk5LWnWxhAMqpchX93EHPLdsfkXSFpD1tFwagvipX0c+XdK/truY/EH4ZEY+0WxaAJlS5iv4Xze8JDmCBYSQbkBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHERppsUlXHjumpVnZF+pDedLkJIJ3ZRcXakqSOZou1NRvlJpvY5V6zRYX7sMNR5rn1+6GI4S8aPTiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJBY5YAP1kZ/zjbrsQELxCg9+E2SdrdVCIDmVd3ZZLmkqyRtarccAE2q2oPfLukWSWW39gBQS5WND9ZLOhAR24Yc97+9ydiaDJgMQ6eL2v6BpOsk9SQtlnS2pAcj4trj/R+mizbUHtNFazvVp4uOuvngFyR9KyLWn+g4At5QewS8tlM94PwdHEiMFV1GQA/eDHrw+ujBARBwIDMCDiRGwIHECDiQGAEHEiPgQGIEHEisldEoIWsuum089Id0ZueKtCNJffWKtSVJmio3badTcp5gTBdrai7KvmbunFamof6hSofRgwOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4lVGslm+0VJb0uak9SLiLVtFgWgGaMMVf1iRBxsrRIAjeMUHUisasBD0q9tb7O9sc2CADSn6in65yLiZdvnSXrC9p6IeOroAwbBJ/zABBl5XXTb35X0TkT86PjHdKLbLbOGuAuuHd5XmSmwR3SmCk6F7ZdbF11RZs18SeoUni4aLvO+n+sdUsTwF63K5oNn2D7ryO+SviTp+folAmhblY/Sj0l6yPaR438REY+1WhWARgwNeETsk/SZArUAaBh/JgMSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kFhLg4JDisPtPPQH9DvltsE5Lcp+Hs72y+0nVGYE9by5Qu8NSeq57Gs2PVdm7Htf1eaQ0IMDiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJVQq47SW2t9jeY3u37UvaLgxAfVWHqv5E0mMR8TXbiySd3mJNABoydF102+dI2iHpE1FxEXXb0e2UOfuPTrk1touPRe+UW/O93Ih+aa4/2lr8dZQfi15mffnD0VM/ov666JJWSXpd0j22n7O9abA+OoAJVyXgU5IuknRHRKyR9K6kWz94kO2Ntrfa3tpwjQBOUpWAz0iaiYhnBre3aD7w7xMRd0bEWvYOBybH0IBHxGuS9ttePbjrckm7Wq0KQCOqXqG6UdLmwRX0fZKub68kAE0ZeXfRSg/KVfRGcBW9Pq6iA0iLgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSKy1YWAuNOrLKjNySJL6LjcCS5IU5caX9frlPuvn4lCxtjrl3h6Syo2cqzCITRI9OJAaAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kNjQgNtebXvHUT9v2b65RHEA6hlp0UXbXUkvS/psRPzjBMfFlAsthtjtlmlHUrmW5vUKtuW0Q1XLjlWNQu+Sfv+wIvqNL7p4uaS/nyjcACbHqN3sNZLuO9Y/2N4oaWPtigA0pvIp+mDTg1ckfToi/jnkWE7RG8Apen2cold3paTtw8INYHKMEvANOs7pOYDJVOkUfbAf+EuSPhERb1Y4nlP0BnCKXt+pfopeKYUR8a6kj9auCkBRjGQDEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQIOJBYW8PNDvaiN+qU0qWSDo7cUq/ceK8aLZ3cc5t8E/+85k5ut6kaz2vu5P7b6D5e5aCRFnxok+2tEbF23HW0Ietz43lNPk7RgcQIOJDYJAX8znEX0KKsz43nNeEm5js4gOZNUg8OoGETEXDb62y/YHuv7VvHXU8TbK+w/aTtXbZ32r5p3DU1yXbX9nO2Hxl3LU2yvcT2Ftt7bO+2fcm4a6pj7Kfog7XW/ybpCkkzkp6VtCEido21sJpsny/p/IjYbvssSdskfXWhP68jbH9D0lpJZ0fE+nHX0xTb90r6XURsGiw0enpEvDHuuk7WJPTgF0vaGxH7ImJW0v2Srh5zTbVFxKsRsX3w+9uSdktaNt6qmmF7uaSrJG0ady1Nsn2OpM9LukuSImJ2IYdbmoyAL5O0/6jbM0oShCNsr5S0RtIz462kMbdLukVSf9yFNGyVpNcl3TP4+rFpsB7hgjUJAU/N9pmSHpB0c0S8Ne566rK9XtKBiNg27lpaMCXpIkl3RMQaSe9KWtDXhCYh4C9LWnHU7eWD+xY829OaD/fmiHhw3PU05FJJX7H9oua/Tl1m++fjLakxM5JmIuLImdYWzQd+wZqEgD8r6QLbqwYXNa6R9PCYa6rNtjX/XW53RNw27nqaEhHfjojlEbFS86/VbyLi2jGX1YiIeE3SfturB3ddLmlBXxQttHj58UVEz/YNkh7X/NLjd0fEzjGX1YRLJV0n6a+2dwzu+05EPDrGmjDcjZI2DzqbfZKuH3M9tYz9z2QA2jMJp+gAWkLAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCCx/wI3IGLE1HB5hgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "primal_mean_abs = np.mean(np.abs(primal_effects), axis=0)\n",
    "plt.imshow(utils.normalize(primal_mean_abs, _range=[0.0, 1.0], _domain=[0.0, shap_values_mean_abs.max()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7faa945bd5f8>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPgAAAD8CAYAAABaQGkdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAADE1JREFUeJzt3V+IXOUdxvHnmdnd/NGo0NoiSaihSMBetJEgFYu0iiXWUHvRCwMKlUKuFKUtYnvVXpTeFLEXRZCoFZoqJSqIWK1QqRVaaxJTav5Y0tSSpNpEiqiJm92Z+fViJxI1cc5kznln9uf3Aws7k5N5f2dnn3nPzL5/HBECkFNr3AUAaA4BBxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQ2FQTDzpjxzI38cinUXAgXk+lTmrBrMud3DkFX+t7iUdPzhV6zuZD6kYM/IVsJODLLH15ukwY2p1yoXtPS4q1JUl727PF2rrc5xZra7b7XrG2SnYAkvSvmW6Rdg6e6FU6jkt0IDECDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJVQq47Q22X7W93/ZdTRcFoB4DA267LemXkq6TdKmkTbYvbbowAKOr0oNfLml/RByIiDlJj0i6odmyANShSsBXSjp4yu1D/fsATLjaJpvY3ixpsyQtretBAYykSg9+WNLqU26v6t/3ARFxX0Ssj4j1M2VnVQI4gyoBf0nSJbbX2J6RdKOkJ5otC0AdBl6iR0TH9q2SnpHUlvRAROxuvDIAI6v0HjwinpL0VMO1AKgZI9mAxAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiTmaGAbGS9rx9Tny0w56R49UaQdSZpeUXabjF63XaytmKu2U0YdWi43WaG3tGwf5tkyz1nnyKxirjfwB0kPDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJFZlZ5MHbB+x/UqJggDUp0oP/itJGxquA0ADBgY8Ip6X9L8CtQCoGe/BgcQa2bpI02xtAkyC2nrwU7cuUpuAA5OAS3QgsSp/JntY0p8lrbV9yPZ3my8LQB2q7E22qUQhAOrHJTqQGAEHEiPgQGIEHEiMgAOJEXAgMQIOJEbAgcQa2bqo3XIsnyrz2jFbcBucdrndfRbaKzimf3bwLji1iW6xprREBRuT1FOZ7a3me6FeBFsXAZ9kBBxIjIADiRFwIDECDiRGwIHECDiQGAEHEiPgQGIEHEisyqKLq20/Z3uP7d22by9RGIDRVdn4oCPp+xGx0/YKSTtsPxsRexquDcCIquxN9npE7Ox//46kvZJWNl0YgNENtXWR7YslrZP04mn+7f2ti9jXBJgMlaeL2j5X0h8l/TQiHvu4Y5kuWlN7TBcdGdNFK7A9LelRSVsHhRvA5KjyKbol3S9pb0Tc3XxJAOpSpQe/UtLNkq62vav/9Y2G6wJQgyp7k70gPjcDFiVGsgGJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSGyo2WRV9SQdb+KBT6MV5V6juoUmEpw0V3B2S6vgZJOlzfzanVanVXiMVqlTOzFf6TB6cCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwILEqiy4utf1X23/rb130kxKFARhdlYF1JyRdHRHv9pdPfsH27yLiLw3XBmBEVRZdDEnv9m9O97/KDsoGcFaqbnzQtr1L0hFJz0bEabcusr3d9nbiD0yGylsXSZLtCyQ9Lum2iHjljMe1HK1CWxe11C7SjiRpiJ9VHbozBWeTzZebdbWkW3I2WeHeptCpzZ+YV683eArgUCmMiLckPSdpw9kWBqCcKp+iX9jvuWV7maRrJe1rujAAo6tyQXGRpIdst7XwgvDbiHiy2bIA1GGo9+CVH5T34LXgPfjoeA8OIC0CDiRGwIHECDiQGAEHEiPgQGIEHEiMgAOJNfJneUtql3rtmC43GMTzS4q1JUntudlibc31yg0+ea89V6ytJb2yfdhctR2FRlZ1gBo9OJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kRsCBxAg4kFjlgPfXRn/ZNuuxAYvEMD347ZL2NlUIgPpV3dlklaTrJW1pthwAdarag98j6U5J5WZ2ABhZlY0PNko6EhE7Bhz3/t5khVcXBnAGA9dFt/0zSTdL6khaKuk8SY9FxE1n+j+tlmNqqtD0w+lyryalp4s6Sk4XnS7WljJPFy3UTq/bVUQMXBd92M0HvyrpBxGx8eOOI+A1tUfAR/ZJDzh/BwcSa2TrInrwmtqjBx8ZPTiAtAg4kBgBBxIj4EBiBBxIjIADiRFwIDECDiTWyGiUiJZ6vWVNPPRH+MTxIu1Ikl1ugIYkddsFB/FMFdpzR1KrU25QTa/VKdaWJFnLyzTUq/Z7Tw8OJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kVmkkm+3XJL0jqSupExHrmywKQD2GGar6tYh4s7FKANSOS3QgsaoBD0m/t73D9uYmCwJQn6qX6F+JiMO2PyPpWdv7IuL5Uw/oB78f/oGruQIoYOh10W3/WNK7EfHzMx/Tjnb7nBFLq1iPyk0XbbnsC1ev3S3XVsFza3UKrZkvqeWy00W7haaLdjvHFb3u6Oui2z7H9oqT30v6uqRXRi8RQNOqvJR+VtLjXniFn5L0m4h4utGqANRiYMAj4oCkLxaoBUDN+DMZkBgBBxIj4EBiBBxIjIADiRFwIDECDiRGwIHEGhoU3JPiWDMP/SHRLrcNzpJe2bHox4ecJzCKmV65trpRbpuk+cJ92Ex3tkg7vehVOo4eHEiMgAOJEXAgMQIOJEbAgcQIOJAYAQcSI+BAYgQcSKxSwG1fYHub7X2299q+ounCAIyu6lDVX0h6OiK+bXtGKrQ2LICRDAy47fMlXSXpO5IUEXOS5potC0Adqlyir5F0VNKDtl+2vaW/PjqACVcl4FOSLpN0b0Ssk3RM0l0fPsj2ZtvbbW+vuUYAZ6lKwA9JOhQRL/Zvb9NC4D8gIu6LiPXsHQ5MjoEBj4g3JB20vbZ/1zWS9jRaFYBaVP0U/TZJW/ufoB+QdEtzJQGoy9C7i1Z6UDvarUJjaAqu6LK89Iou7XIrn0wXXD2mW3D1mI5Lr+hS5ndkLjrqRYy+uyiAxYuAA4kRcCAxAg4kRsCBxAg4kBgBBxIj4EBiBBxIrKG9ySzHkmYe+kNaFfdoqsOc28XakiQVHPHVKzhKr6NyI/Smij9lhRrsdisdRg8OJEbAgcQIOJAYAQcSI+BAYgQcSIyAA4kRcCAxAg4kNjDgttfa3nXK19u27yhRHIDRDByqGhGvSvqSJNluSzos6fGG6wJQg2Ev0a+R9M+I+HcTxQCo17CTTW6U9PDp/sH2Zkmb+7dGKgpAPSqvi97f9OA/kr4QEf/9+GNbMeWlNZQ3WKtdbjaZm5p8dwYddYq11S44m2y+5Gyysk+Z1C3TYKc7p14MftKGuUS/TtLOQeEGMDmGCfgmneHyHMBkqhTw/n7g10p6rNlyANSp0huGiDgm6VMN1wKgZoxkAxIj4EBiBBxIjIADiRFwIDECDiRGwIHECDiQWOXJJkM9qH1U0rBTSj8t6c3ai5kMWc+N8xqfz0XEhYMOaiTgZ8P29ohYP+46mpD13DivycclOpAYAQcSm6SA3zfuAhqU9dw4rwk3Me/BAdRvknpwADWbiIDb3mD7Vdv7bd817nrqYHu17eds77G92/bt466pTrbbtl+2/eS4a6mT7Qtsb7O9z/Ze21eMu6ZRjP0Svb/W+j+0sGLMIUkvSdoUEXvGWtiIbF8k6aKI2Gl7haQdkr612M/rJNvfk7Re0nkRsXHc9dTF9kOS/hQRW/oLjS6PiLfGXdfZmoQe/HJJ+yPiQETMSXpE0g1jrmlkEfF6ROzsf/+OpL2SVo63qnrYXiXpeklbxl1LnWyfL+kqSfdLUkTMLeZwS5MR8JWSDp5y+5CSBOEk2xdLWifpxfFWUpt7JN0pqdya1WWskXRU0oP9tx9b+usRLlqTEPDUbJ8r6VFJd0TE2+OuZ1S2N0o6EhE7xl1LA6YkXSbp3ohYJ+mYpEX9mdAkBPywpNWn3F7Vv2/Rsz2thXBvjYgsK9JeKembtl/Twtupq23/erwl1eaQpEMRcfJKa5sWAr9oTULAX5J0ie01/Q81bpT0xJhrGplta+G93N6IuHvc9dQlIn4YEasi4mItPFd/iIibxlxWLSLiDUkHba/t33WNpEX9oWjpjV0+IiI6tm+V9IyktqQHImL3mMuqw5WSbpb0d9u7+vf9KCKeGmNNGOw2SVv7nc0BSbeMuZ6RjP3PZACaMwmX6AAaQsCBxAg4kBgBBxIj4EBiBBxIjIADiRFwILH/A55PNa3EkUYQAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "interactions_mean_abs = np.mean(np.abs(shap_values - primal_effects), axis=0)\n",
    "plt.imshow(utils.normalize(interactions_mean_abs, _range=[0.0, 1.0], _domain=[0.0, shap_values_mean_abs.max()]))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
